Retry_dir=Retry
if [ ! -d $Retry_dir ]; then mkdir $Retry_dir; fi

timetag=`date +%Y%m%d%H%M`
check_jobID=-1
nFinished=-1
checkFailedID() {
    pbook -c "show($1)"  > tmp_status
    jobStatus=tmp_status
    nFailed=`less $jobStatus | grep failed | awk '{print $3}'`
    nCancelled=`less $jobStatus | grep cancelled | awk '{print $3}'`
    nFinished=`less $jobStatus | grep finished | awk '{print $3}'`
    if [ -z "${nFinished}" ]; then nFinished=0; fi
    echo "failed: $nFailed; cancelled: $nCancelled; finished: $nFinished"
    if [ -n "${nFailed}" ] || [ -n "${nCancelled}" ]; then
        retrySetID=`less $jobStatus | grep retrySetID | awk '{print $3}'`
        retryID=`less $jobStatus | grep retryID | awk '{print $3}'`
        if [ -n "${retryID}" ] ; then
            checkFailedID $retryID
        elif [ -n "${retrySetID}" ] ; then
            checkFailedID $retrySetID
        else 
            check_jobID=$1
        fi
    else 
        check_jobID=-1    
    fi

}

pbook -c "sync()"
pbook -c "show(5458)" > $Retry_dir/out_frozen_$timetag
less $Retry_dir/out_frozen_$timetag | grep JobsetID | awk '{ print $3 }'  > $Retry_dir/frozen_jobID_$timetag

for job in `cat $Retry_dir/frozen_jobID_$timetag`; do
    check_jobID=$job
    nFinished=-1
    checkFailedID $job
    echo "failed_jobID: $check_jobID"
    if [ "${check_jobID}" == "-1" ]; then
        echo "Succeeded: $job" >> $Retry_dir/good_job_$timetag
    else 
        echo "Failed: $check_jobID" >> $Retry_dir/failed_job_$timetag
    fi
done




